# Title: Pause Long Running Graphics Job # Description: Freezes a render, export, transcode or similar job in place without # closing it and without losing completed work # # No double quotes and no semicolons are used anywhere in this file. The agent # script runner wraps content in double quotes, so an inner double quote ends # the wrapper early and any following text is run as a command. The interop # here-string below needs both characters, so they are injected via [char]34 # and [char]59 through the format operator rather than typed literally. This # file is checked by resources/build/validate_script_quoting.py. # Batch workloads that tolerate being frozen and later carried on # Every name here is a guess and must be confirmed against the real fleet # Known miner binaries have been removed from this list on purpose - a miner is a # security signal, not a batch job, and is handled by the incident path below $pausableBatchProcesses = @( 'ffmpeg', 'HandBrakeCLI', 'HandBrake', 'Adobe Media Encoder', 'blender', 'vray', 'deadlineworker', 'deadlineslave', '3dsmaxcmd' ) # Known miner binaries. Their presence is a compromise signal and is reported, never # paused. A paused miner resumes the moment the interactive session ends, so a silent # pause would mask a live infection and burn the one signal that would surface it. # Detection here does not require a GPU sample, because a miner can idle on the GPU or # mine on the CPU and still be a live threat worth reporting $knownMinerProcesses = @( 'xmrig', 't-rex', 'nbminer', 'lolMiner', 'gminer', 'phoenixminer', 'ethminer', 'teamredminer', 'NiceHashMiner' ) # Engine points on one engine of one adapter before a batch job is worth pausing $pauseThreshold = 25 # Durable audit sink for the elevated action this lever takes and for miner alerts # The event log write is best effort - a blocked or missing source degrades to # standard output and never stops the remediation. Swap this one function if the # fleet standard is a different sink $dexAuditSource = 'DexGpuRemediation' function Write-RemediationEvent { param( [int]$EventId, [string]$EntryType, [string]$Message ) Write-Output $Message try { if (-not [System.Diagnostics.EventLog]::SourceExists($dexAuditSource)) { New-EventLog -LogName 'Application' -Source $dexAuditSource -ErrorAction Stop } Write-EventLog -LogName 'Application' -Source $dexAuditSource -EventId $EventId -EntryType $EntryType -Message $Message -ErrorAction Stop } catch { Write-Output ('audit: event log write failed - {0}' -f $_.Exception.Message) } } # Incident path - report any known miner before any performance work is attempted # This runs even when the GPU counters are unavailable, so detection is never gated on # a sample succeeding. The full path and hash are captured for downstream triage $detectedMiners = @(Get-Process -Name $knownMinerProcesses -ErrorAction SilentlyContinue) foreach ($minerProcess in $detectedMiners) { $minerPath = 'unavailable' $minerHash = 'unavailable' try { $resolvedMinerPath = $minerProcess.Path if (-not [string]::IsNullOrWhiteSpace($resolvedMinerPath)) { $minerPath = $resolvedMinerPath $minerHash = (Get-FileHash -Path $resolvedMinerPath -Algorithm SHA256 -ErrorAction Stop).Hash } } catch { $minerHash = 'unavailable' } Write-RemediationEvent -EventId 4210 -EntryType 'Warning' -Message ('alert: suspected miner {0} pid {1} path {2} sha256 {3} routed to incident path and not paused' -f $minerProcess.ProcessName, $minerProcess.Id, $minerPath, $minerHash) } $engineSampleSet = Get-Counter -Counter '\GPU Engine(*)\Utilization Percentage' -SampleInterval 1 -MaxSamples 2 -ErrorAction SilentlyContinue if ($null -eq $engineSampleSet) { Write-Output 'skip: gpu engine counters unavailable on this device' return } # Sum engine indexes that share a physical adapter and an engine type, then take the # heaviest single engine per process so the unit stays absolute on a multi adapter box $engineTotals = @{} foreach ($engineSample in $engineSampleSet[-1].CounterSamples) { if ($engineSample.CookedValue -le 0) { continue } if ($engineSample.InstanceName -notmatch 'pid_(\d+)_.*phys_(\d+)_eng_\d+_engtype_(.+)$') { continue } $engineBucketKey = '{0}|{1}|{2}' -f $Matches[1], $Matches[2], $Matches[3] if ($engineTotals.ContainsKey($engineBucketKey)) { $engineTotals[$engineBucketKey] = $engineTotals[$engineBucketKey] + $engineSample.CookedValue } else { $engineTotals[$engineBucketKey] = $engineSample.CookedValue } } $peakLoadByProcessId = @{} foreach ($engineBucketKey in $engineTotals.Keys) { $bucketOwnerProcessId = [int]($engineBucketKey -split '\|')[0] $bucketLoad = $engineTotals[$engineBucketKey] if ((-not $peakLoadByProcessId.ContainsKey($bucketOwnerProcessId)) -or ($bucketLoad -gt $peakLoadByProcessId[$bucketOwnerProcessId])) { $peakLoadByProcessId[$bucketOwnerProcessId] = $bucketLoad } } # The process start time is captured alongside the winner so the identity can be # re-verified just before the elevated action, closing the pid reuse window $pauseTarget = $null $pauseTargetLoad = 0 $pauseTargetStartTime = $null foreach ($candidateProcessId in $peakLoadByProcessId.Keys) { if ($peakLoadByProcessId[$candidateProcessId] -lt $pauseThreshold) { continue } if ($candidateProcessId -eq $PID) { continue } $candidateProcess = Get-Process -Id $candidateProcessId -ErrorAction SilentlyContinue if ($null -eq $candidateProcess) { continue } if ($pausableBatchProcesses -notcontains $candidateProcess.ProcessName) { continue } if ($peakLoadByProcessId[$candidateProcessId] -gt $pauseTargetLoad) { $pauseTarget = $candidateProcess $pauseTargetLoad = $peakLoadByProcessId[$candidateProcessId] $pauseTargetStartTime = $null try { $pauseTargetStartTime = $candidateProcess.StartTime } catch { $pauseTargetStartTime = $null } } } if ($null -eq $pauseTarget) { Write-Output ('skip: no allowlisted batch job above {0} engine points' -f $pauseThreshold) return } # A fully frozen process has every thread waiting on a suspend, which makes this # check the idempotency guard and removes any need for stored state on disk $activeThreadCount = @($pauseTarget.Threads | Where-Object { $_.WaitReason -ne 'Suspended' }).Count if ($activeThreadCount -eq 0) { Write-Output ('skip: {0} already paused' -f $pauseTarget.ProcessName) return } # Guard against pid reuse between sampling and action. If the sampled process exited # and Windows recycled its pid onto a different process, the identity below will not # match and the suspend is abandoned rather than aimed at the wrong process $recheckProcess = Get-Process -Id $pauseTarget.Id -ErrorAction SilentlyContinue $recheckStartTime = $null if ($null -ne $recheckProcess) { try { $recheckStartTime = $recheckProcess.StartTime } catch { $recheckStartTime = $null } } if ($null -eq $recheckProcess -or $recheckProcess.ProcessName -ne $pauseTarget.ProcessName -or $recheckStartTime -ne $pauseTargetStartTime) { Write-Output 'skip: pause target identity changed before action' return } # Full image path and file hash are resolved for the audit record. Acting on a process # name alone is spoofable, so this leaves a forensic trail even when the name matches # an expected batch job. A path allowlist is a fleet specific follow up and is not # enforced here, because legitimate render and transcode tools are often unsigned $pauseTargetPath = 'unavailable' $pauseTargetHash = 'unavailable' try { $resolvedPath = $recheckProcess.Path if (-not [string]::IsNullOrWhiteSpace($resolvedPath)) { $pauseTargetPath = $resolvedPath $pauseTargetHash = (Get-FileHash -Path $resolvedPath -Algorithm SHA256 -ErrorAction Stop).Hash } } catch { $pauseTargetHash = 'unavailable' } # Authenticode status and signer are captured for the audit trail on every action. # A process name alone is spoofable, and an unsigned binary is not by itself # suspicious (many legitimate render/transcode tools ship unsigned), so this is not # used as a gate in the common case. HashMismatch is different - it means the # on-disk bytes were altered after signing, which is a tamper signal rather than a # policy choice, so that specific status blocks the elevated action and is routed # to the audit log as a warning instead $pauseTargetSignatureStatus = 'unavailable' $pauseTargetSignerSubject = 'unavailable' try { if (-not [string]::IsNullOrWhiteSpace($pauseTargetPath)) { $pauseTargetSignature = Get-AuthenticodeSignature -FilePath $pauseTargetPath -ErrorAction Stop $pauseTargetSignatureStatus = $pauseTargetSignature.Status.ToString() if ($null -ne $pauseTargetSignature.SignerCertificate) { $pauseTargetSignerSubject = $pauseTargetSignature.SignerCertificate.Subject } } } catch { $pauseTargetSignatureStatus = 'unavailable' } if ($pauseTargetSignatureStatus -eq 'HashMismatch') { Write-RemediationEvent -EventId 4211 -EntryType 'Warning' -Message ('alert: signature hash mismatch for {0} pid {1} path {2} sha256 {3} - possible tampered binary, pause skipped' -f $pauseTarget.ProcessName, $pauseTarget.Id, $pauseTargetPath, $pauseTargetHash) return } # The interop source needs quote and semicolon characters that cannot appear in # this file, so they are supplied through the format operator $processControlMembers = @' [System.Runtime.InteropServices.DllImport({0}ntdll.dll{0})] public static extern int NtSuspendProcess(System.IntPtr processHandle){1} [System.Runtime.InteropServices.DllImport({0}kernel32.dll{0})] public static extern System.IntPtr OpenProcess(int desiredAccess, bool inheritHandle, int processId){1} [System.Runtime.InteropServices.DllImport({0}kernel32.dll{0})] public static extern bool CloseHandle(System.IntPtr objectHandle){1} '@ -f [char]34, [char]59 if (-not ('DexProcessControl.Freeze' -as [type])) { Add-Type -Namespace 'DexProcessControl' -Name 'Freeze' -MemberDefinition $processControlMembers -ErrorAction Stop } # 0x0800 is suspend and resume access $targetProcessHandle = [DexProcessControl.Freeze]::OpenProcess(0x0800, $false, $pauseTarget.Id) if ($targetProcessHandle -eq [System.IntPtr]::Zero) { Write-Output ('skip: no access to {0}' -f $pauseTarget.ProcessName) return } # A frozen process can hold submitted GPU packets and unsignalled fences, which in # rare cases trips a display driver timeout and resets the adapter $suspendStatus = [DexProcessControl.Freeze]::NtSuspendProcess($targetProcessHandle) [void][DexProcessControl.Freeze]::CloseHandle($targetProcessHandle) if ($suspendStatus -eq 0) { Write-RemediationEvent -EventId 4201 -EntryType 'Information' -Message ('paused: {0} pid {1} at {2} engine points path {3} sha256 {4} signature {5} signer {6}' -f $pauseTarget.ProcessName, $pauseTarget.Id, [math]::Round($pauseTargetLoad, 0), $pauseTargetPath, $pauseTargetHash, $pauseTargetSignatureStatus, $pauseTargetSignerSubject) } else { Write-Output ('skip: pause refused for {0}' -f $pauseTarget.ProcessName) }